
                /*********************************************
                **                                          **
                **                                          **
                **               sa.op.cml.C                **
                **                                          **
                **                                          **
                *********************************************/
                /*********************************************
                **                                          **
                **         Author Jean-Marc Adamo           **
                **       Copyright (C) 1997-2008 JMA        **
                *********************************************/


/*
** Spread Arrays and 
** Non-Blocking Global Read/Write Functions
** Needs 4 processors
*/

#include "arch.h"

Global_id gid;
Coll_id cid, cid1, cid2;

void init_gid(int, char **){
   gid.set();
   cid.set();
   cid1.set();
   cid2.set();
   fprintf(Sdf::trace, "from node %d: gid1=%d, cid=%d, cid1=%d, cid2=%d\n",
           Sdf::PROC_ID, gid.get(), cid.get(), cid1.get(), cid2.get());
   fflush(Sdf::trace);
}

#define d0 4
#define d1 5
#define d2 2

class RW_count{
   public:
      int cnt;

      RW_count(){
         cnt = 0;
      }
      void reset(){
         cnt = 0;
      }
      void more_RW(int i){
         cnt += i;
      }
};

#define wait_while(condition) while(condition) Sdf::poll();

#define RW_not_complete() (count->cnt < d0*d1*d2)

RW_count *count;

void upon_end_of_RW(HRW_rcd *){
   count->cnt++;
}

class Root_thread: public Thread{
   public:
        Root_thread()
                : Thread((Tmf0_ptr)&Root_thread::body){}

        void body(){

           int colour;
           if(Sdf::PROC_ID==Sdf::PROC_NB-1)
              colour=Sdf::UNDEFINED;
           else
              colour=1;

             //Sdf::UNDEFINED is not counted as a colour
           CommL **comm_table = 
           Sdf::COMML_WORLD->split(cid, 1, colour, Sdf::PROC_NB-2-Sdf::PROC_ID);
           CommL *comm = *comm_table;

           test_code(comm);

           delete comm;
        }


        void test_code(CommL *comm){

           int comm_rank = comm->rank();
           int comm_size = comm->size();

fprintf(Sdf::trace, "node%d:>>>1 comm=%d, rank=%d, size=%d, Sdf::UNDEFINED=%d\n", 
        Sdf::PROC_ID, 
        comm,
        comm_rank,
        comm_size,
        Sdf::UNDEFINED);
fflush(Sdf::trace);

             //access space: Sdf::COMML_WORLD
             //definition space: comm
           SpreadArray<int> SA(gid, d0, d1, d2, 2, Sdf::COMML_WORLD, comm);

fprintf(Sdf::trace, "node%d:>>>2 rank=%d, p_id(wrt comm)=%d, offset=%d, comm=%d\n", 
        Sdf::PROC_ID, 
        comm_rank,
        SA.get_proc_id(),
        SA.get_offset(),
        SA.get_comm());
fflush(Sdf::trace);

           if(Sdf::PROC_ID==Sdf::PROC_NB-1){

fprintf(Sdf::trace, "node%d: before flush1\n", Sdf::PROC_ID);
fflush(Sdf::trace);

              Sdf::barrier(cid1);
                //Synchronizes last process with the processes in comm 
                //so that initialization they performed is available.

              fprintf(Sdf::trace, "node%d: !!!!!!!!!!1\n", Sdf::PROC_ID);
              fflush(Sdf::trace);
              int data;
              int i;
              for(i=0; i<d0; i++)
                 for(int j = 0; j < d1; j++)
                    for(int k = 0; k < d2; k++){
                       SA(i,j,k).read(data);
                       fprintf(Sdf::trace, "node%d: data=%d\n", Sdf::PROC_ID, data);
                       fflush(Sdf::trace);
                    }

              Star<int> SS = SA;
              Ref<int> SSS = (*SS);

fprintf(Sdf::trace, "node%d:>>>3 rank=%d, p_id=%d, tag_ev=%d, offset=%d, comm=%d\n",
        Sdf::PROC_ID,
        comm_rank,
        SS.get_proc_id(),
        SS.get_tag_ev(),
        SS.get_offset(),
        SS.get_comm());
fflush(Sdf::trace);
fprintf(Sdf::trace, "node%d:>>>4 rank=%d, p_id(wrt comm)=%d, tag_ev=%d, offset=%d, comm=%d\n",
        Sdf::PROC_ID,
        comm_rank,
        SSS.get_proc_id(),
        SSS.get_tag_ev(),
        SSS.get_offset(),
        SSS.get_comm());
fflush(Sdf::trace);

              i = 0;
              while(1){
                 (*SS).read(data);
                 fprintf(Sdf::trace, "node%d: data=%d\n", Sdf::PROC_ID, data);
                 fflush(Sdf::trace);
                 if(i<d0*d1*d2-1){
                    SS++;
                    i++;
                 }else
                    break;
              }

fprintf(Sdf::trace, "node%d:>>>5 rank=%d, p_id(wrt comm)=%d, tag_ev=%d, offset=%d, comm=%d\n",
        Sdf::PROC_ID,
        comm_rank,
        SSS.get_proc_id(),
        SSS.get_tag_ev(),
        SSS.get_offset(),
        SSS.get_comm());
fflush(Sdf::trace);
              i = 0;
              while(1){
                 (*SS).read(data);
                 fprintf(Sdf::trace, "node%d: data=%d\n", Sdf::PROC_ID, data);
                 fflush(Sdf::trace);
                 if(i<d0*d1*d2-1){
                    SS--;
                    i++;
                 }else
                    break;
              }

fprintf(Sdf::trace, "node%d:>>>6 rank=%d, p_id(wrt comm)=%d, tag_ev=%d, offset=%d, comm=%d\n",
        Sdf::PROC_ID,
        comm_rank,
        SSS.get_proc_id(),
        SSS.get_tag_ev(),
        SSS.get_offset(),
        SSS.get_comm());
fflush(Sdf::trace);

              i = 0;
              while(1){
                 (*SS).read(data);
                 fprintf(Sdf::trace, "node%d: data=%d\n", Sdf::PROC_ID, data);
                 fflush(Sdf::trace);
                 if(i<d0*d1*d2-1){
                    ++SS;
                    i++;
                 }else
                    break;
              }

fprintf(Sdf::trace, "node%d:>>>7 rank=%d, p_id(wrt comm)=%d, tag_ev=%d, offset=%d, comm=%d\n",
        Sdf::PROC_ID,
        comm_rank,
        SSS.get_proc_id(),
        SSS.get_tag_ev(),
        SSS.get_offset(),
        SSS.get_comm());
fflush(Sdf::trace);

              i = 0;
              while(1){
                 (*SS).read(data);
                 fprintf(Sdf::trace, "node%d: data=%d\n", Sdf::PROC_ID, data);
                 fflush(Sdf::trace);
                 if(i<d0*d1*d2-1){
                    --SS;
                    i++;
                 }else
                    break;
              }

fprintf(Sdf::trace, "node%d:>>>8 rank=%d, p_id(wrt comm)=%d, tag_ev=%d, offset=%d, comm=%d\n",
        Sdf::PROC_ID,
        comm_rank,
        SSS.get_proc_id(),
        SSS.get_tag_ev(),
        SSS.get_offset(),
        SSS.get_comm());
fflush(Sdf::trace);

              i = 0;
              while(1){
                 (*SS).read(data);
                 fprintf(Sdf::trace, "node%d: data=%d\n", Sdf::PROC_ID, data);
                 fflush(Sdf::trace);
                 if(i<d0*d1*d2-1){
                    SS+=1;
                    i++;
                 }else
                    break;
              }

fprintf(Sdf::trace, "node%d:>>>9 rank=%d, p_id(wrt comm)=%d, tag_ev=%d, offset=%d, comm=%d\n",
        Sdf::PROC_ID,
        comm_rank,
        SSS.get_proc_id(),
        SSS.get_tag_ev(),
        SSS.get_offset(),
        SSS.get_comm());
fflush(Sdf::trace);

              i = 0;
              while(1){
                 (*SS).read(data);
                 fprintf(Sdf::trace, "node%d: data=%d\n", Sdf::PROC_ID, data);
                 fflush(Sdf::trace);
                 if(i<d0*d1*d2-1){
                    SS-=1;
                    i++;
                 }else
                    break;
              }

fprintf(Sdf::trace, "node%d:>>>10 rank=%d, p_id(wrt comm)=%d, tag_ev=%d, offset=%d, comm=%d\n",
        Sdf::PROC_ID,
        comm_rank,
        SSS.get_proc_id(),
        SSS.get_tag_ev(),
        SSS.get_offset(),
        SSS.get_comm());
fflush(Sdf::trace);

              i = 0;
              while(1){
                 (*SS).read(data);
                 fprintf(Sdf::trace, "node%d: data=%d\n", Sdf::PROC_ID, data);
                 fflush(Sdf::trace);
                 if(i<d0*d1*d2-1){
                    SS=SS+1;
                    i++;
                 }else
                    break;
              }

fprintf(Sdf::trace, "node%d:>>>11 rank=%d, p_id(wrt comm)=%d, tag_ev=%d, offset=%d, comm=%d\n",
        Sdf::PROC_ID,
        comm_rank,
        SSS.get_proc_id(),
        SSS.get_tag_ev(),
        SSS.get_offset(),
        SSS.get_comm());
fflush(Sdf::trace);

              i = 0;
              while(1){
                 (*SS).read(data);
                 fprintf(Sdf::trace, "node%d: data=%d\n", Sdf::PROC_ID, data);
                 fflush(Sdf::trace);
                 if(i<d0*d1*d2-1){
                    SS=SS-1;
                    i++;
                 }else
                    break;
              }

              Sdf::barrier(cid2);
                //Synchronizes last process with the processes in comm 
                //before returning.

              return;
           }

             //only processes in comm execute the sequence below
           int *local_ptr = SA.to_local_ptr();
           if(!comm_rank){
              local_ptr[0] =  0;     local_ptr[1] =    1;
              local_ptr[2] =  30;    local_ptr[3] =   31;
              local_ptr[4] =  110;   local_ptr[5] =  111;
              local_ptr[6] =  140;   local_ptr[7] =  141;
              local_ptr[8] =  220;   local_ptr[9] =  221;
              local_ptr[10] = 300;   local_ptr[11] = 301;
              local_ptr[12] = 330;   local_ptr[13] = 331;
           }else
              if(comm_rank == 1){
                 local_ptr[0] =  10;    local_ptr[1] =   11;
                 local_ptr[2] =  40;    local_ptr[3] =   41;
                 local_ptr[4] =  120;   local_ptr[5] =  121;
                 local_ptr[6] =  200;   local_ptr[7] =  201;
                 local_ptr[8] =  230;   local_ptr[9] =  231;
                 local_ptr[10] = 310;   local_ptr[11] = 311;
                 local_ptr[12] = 340;   local_ptr[13] = 341;
              }else{
                 local_ptr[0] =  20;    local_ptr[1] =   21;
                 local_ptr[2] =  100;   local_ptr[3] =  101;
                 local_ptr[4] =  130;   local_ptr[5] =  131;
                 local_ptr[6] =  210;   local_ptr[7] =  211;
                 local_ptr[8] =  240;   local_ptr[9] =  241;
                 local_ptr[10] = 320;   local_ptr[11] = 321;
                 local_ptr[12] = 555;   local_ptr[13] = 555;
              }

fprintf(Sdf::trace, "node%d: before Sdf::flush()2\n", Sdf::PROC_ID);
fflush(Sdf::trace);

           Sdf::barrier(cid1);
                //Synchronizes the processes in comm with last process 
                //so that the latter can apply Global pointer operations
                //on the data above created.

           fprintf(Sdf::trace, "node%d: !!!!!!!!!!2\n", Sdf::PROC_ID);
           fflush(Sdf::trace);

           Sdf::barrier(cid2);
                //Synchronizes the processes in comm with last process 
                //before returning.

           fprintf(Sdf::trace, "node%d:  the end\n\n\n", Sdf::PROC_ID);
           fflush(Sdf::trace);
        }
};

MAIN(Root_thread, init_gid, Sdf::empty, Sdf::empty)
